/* 
 * File:   FileCached.cpp
 * Author: dell
 * 
 * Created on 2012年4月15日, 下午11:19
 */

#include "FileCached.h"
#include <stdlib.h>
#include <string>
#include "StringUtils.h"
#include <sys/stat.h>
#include <sys/types.h>
#include <utime.h>
#include <cerrno>
#include <vector>
#include <dirent.h>
#include "RapidMain.h"
#include "ServerPid.h"
#include <string>

using namespace std;

namespace Rapid{
    FileCached::FileCached(ServerConfig* serverConfig):
        serverConfig(serverConfig), cacheConfig(&serverConfig->cacheConfig), cahcedCapacity(0.0) {
        initAsyncClean();
    }

//    FileCached::FileCached(){
//    }
    
    FileCached::~FileCached() {
        LOG_DEBUG("")
        stopAsyncClean();
        delAllSourceFileEvstats();
        Filenames::iterator it = filenames.begin();
        for(; it != filenames.end(); it++){
            free(it->second);
        }
        filenames.clear();
//        
//        CahcedCapacitys::iterator ccit = cahcedCapacitys.begin();
//        for(; ccit != cahcedCapacitys.end(); ccit++){
//            delete ccit->second;
//        }
//        cahcedCapacitys.clear();
    }

    /** 检查文件是否在cache中 */
    bool FileCached::exist(const char* filename){
        string physicalFileName;
        makePhysicalFileName(physicalFileName, filename);
//        LOG_DEBUG("physicalFileName:%s", physicalFileName.c_str());
        struct stat filestat;
        if(stat(physicalFileName.c_str(), &filestat)!=0){
            LOG_DEBUG("file not exist! errno:%d", errno);
            return false;
        }

        if(S_ISDIR(filestat.st_mode)){
            LOG_DEBUG("is dir, file.stat.st_size:%d", filestat.st_size);
            return false;
        }
        
        return true;
    }

    /** 从cache打开文件FD, 如果文件不存在，返回-1 */
    FD FileCached::openFile(struct stat& filestat, const char* filename){
        string physicalFileName;
        makePhysicalFileName(physicalFileName, filename);
//        LOG_DEBUG("physicalFileName:%s", physicalFileName.c_str());
        if(stat(physicalFileName.c_str(), &filestat)!=0){
            LOG_DEBUG("file not exist! errno:%d", errno);
            return -1;
        }

        if(S_ISDIR(filestat.st_mode)){
            LOG_DEBUG("is dir, file.stat.st_size:%d", filestat.st_size);
            return -1;
        }
        
        //打开文件
        FD fd = ::open(physicalFileName.c_str(), O_RDONLY | O_NONBLOCK);
        if(fd < 0){
            LOG_DEBUG("open file error, errno:%d", errno);
            return -1;
        }
        return fd;
    }

    /** 存储文件 */
    bool FileCached::storeFile(const char* physicalFilename){
//        LOG_DEBUG("physicalFilename:%s", physicalFilename);
        char* fullfilename = StringUtils::replace(physicalFilename, '\\', '/');
        char* rsp = strrchr(fullfilename, '/');
        *rsp = 0;
        char* filepath = fullfilename;
        char* filename = rsp+1;
        LOG_DEBUG("filepath:%s", filepath);
        LOG_DEBUG("filename:%s", filename);
        bool rs = true;
        string shmdir(cacheConfig->base);
        shmdir.append(filepath);
        if(::access(shmdir.c_str(), R_OK | W_OK)!=0){
            if(!makeDirs(filepath)){
                rs = false;
            }
        }
        if(rs){
            //将文件转送到缓存目录
            string physicalFilename(filepath);
            physicalFilename.append("/").append(filename);
            rs = preTransportFile(shmdir.append("/").append(filename), physicalFilename);
        }
        ::free(fullfilename);
        return rs;
    }
    
    
    bool FileCached::preTransportFile(string& shmFilename, string& physicalFilename){
        LOG_DEBUG("shmFilename:%s", shmFilename.c_str());
        LOG_DEBUG("physicalFilename:%s", physicalFilename.c_str());
        
        //物理文件状态
        struct stat pfilestat;
        if(stat(physicalFilename.c_str(), &pfilestat)!=0){
            LOG_WARN("physical file not exist! errno:%d", errno);
            return true;
        }
        
        //检查shmFilename文件是否存在
        struct stat shmfilestat;
        if(stat(shmFilename.c_str(), &shmfilestat)==0){
            //如果文件存在，检查文件大小和时间是否与源文件相同
            if(pfilestat.st_size == shmfilestat.st_size && pfilestat.st_mtim.tv_sec == shmfilestat.st_mtim.tv_sec && pfilestat.st_mtim.tv_nsec == shmfilestat.st_mtim.tv_nsec){
                LOG_WARN("physical file and shm file is same");
                return true;   
            }
        }
        
        //打开文件
        FD pfd = ::open(physicalFilename.c_str(), O_RDONLY | O_NONBLOCK);
        if(pfd < 0){
            LOG_WARN("physical open file error, errno:%d", errno);
            return true;
        }
        
        string tmpShmFilename(shmFilename);
        tmpShmFilename.append(".tmp~~~");
        if(::access(tmpShmFilename.c_str(), F_OK)==0){
            LOG_DEBUG("tmp shm file is exist");
            return true;
        }
        
        FD shmfd = ::open(tmpShmFilename.c_str(), O_RDWR | O_CREAT | O_TRUNC | O_NONBLOCK, S_IRWXU | S_IRWXG | S_IRWXO);
        if(shmfd < 0){
            LOG_WARN("shm create tmp file error, errno:%d", errno);
            return true;
        }        
        
        SendfileItem* item = new SendfileItem();
        item->basepath = cacheConfig->base;
        item->physicalFilename = physicalFilename;
        item->shmFilename = shmFilename;
        item->tmpShmFilename = tmpShmFilename;
        item->srcfd = pfd;
        item->shmfd = shmfd;
        item->pfilestat = pfilestat;
        item->ioreader = new EvNIOReader(pfd);
        item->iowriter = new EvNIOWriter(shmfd);
        item->sendfile = new RWIOPump(item->ioreader, item->iowriter, 10485760, 1048576);
        item->sendfile->addListener(EVENT_TYPE_IO_TRANSPORT, this);
        
        sendfileItems.insert(SendfileItems::value_type(item->sendfile, item));
//        sendfileItems.push_back(item);
        LOG_DEBUG("this:%ld, sendfileItems:%ld, size:%d, %ld", this, &this->sendfileItems, sendfileItems.size(), item->sendfile);
        return item->sendfile->start();
        
    }
    
    
    
    
    void FileCached::makePhysicalFileName(std::string& physicalFileName, const char* filename){
        physicalFileName.append(cacheConfig->base).append(filename);
    }
    
    
    bool FileCached::makeDirs(const char* physicalFilename){
        bool rs = true;
        LOG_DEBUG("physicalFilename:%s", physicalFilename);
//        char* filename = StringUtils::replace(physicalFilename, '\\', '/');
//        LOG_DEBUG("filename:%s", filename);
        vector<const char*> dirnames;
        char* filenamedup = StringUtils::split(dirnames, physicalFilename, '/');
        string dirname(cacheConfig->base);
        LOG_DEBUG("root dir is:%s", dirname.c_str());
        int dirsize = dirnames.size();
        for(int i=0; i<dirsize; i++){
            if(strlen(dirnames[i]) > 0){
                dirname.append("/").append(dirnames[i]);
                LOG_DEBUG("dir is:%s", dirname.c_str());
                if(::access(dirname.c_str(), R_OK | W_OK)!=0){
                    LOG_DEBUG("access rs not is 0");
                    if(mkdir(dirname.c_str(), 0777)==-1){  
                        LOG_ERROR("mkdir failure, dir is %s", dirname.c_str());
                        rs = false;
                        break;
                    }  
                }else{
                    LOG_DEBUG("access rs is 0");   
                }
                
            }
        }
        
        ::free(filenamedup);
//        ::free(filename);
        return rs;
    }
    
    
    
    void FileCached::asyncCallback(ev::async& async, int revents){
        LOG_DEBUG("this:%ld, sendfileItems.size:%d, doneRWIOPumps.size():%d", this, this->sendfileItems.size(), doneRWIOPumps.size());
        DoneRWIOPumps::iterator keyit = doneRWIOPumps.begin();
        for(;keyit != doneRWIOPumps.end(); keyit++){
            RWIOPump* sendfile = *keyit;
            SendfileItems::iterator it = sendfileItems.find(sendfile);
            LOG_DEBUG("it != sendfileItems.end():%d", it != sendfileItems.end());
            if(it != sendfileItems.end()){
                SendfileItem* item = it->second;
                LOG_DEBUG("item->shmfd:%d, item->srcfd:%d", item->shmfd, item->srcfd);
                sendfileItems.erase(it);
                delete item->ioreader;
                delete item->iowriter;
//                delete item->sendfile;
                ::close(item->shmfd);
                ::close(item->srcfd);
                if(::rename(item->tmpShmFilename.c_str(), item->shmFilename.c_str()) !=0){
                    LOG_WARN("rename tmp shm file error, errno:%d", errno);
                    ::remove(item->tmpShmFilename.c_str());
                }
                
                //设置访问时间和修改时间
                struct timezone tz;
                struct timeval times[2];
                gettimeofday(&times[0], &tz);
                times[1].tv_sec = item->pfilestat.st_mtim.tv_sec;
                times[1].tv_usec = item->pfilestat.st_mtim.tv_nsec * 1000;
                ::utimes(item->shmFilename.c_str(), times);
                
                //newSourceFileEvstat(item->shmFilename.c_str());
                if(processMessage.isClosed()){
                    processMessage.open(RapidMain::homeDir.c_str());
                }
                if(!processMessage.isClosed()){
                    LOG_DEBUG("serverConfig->port:%d", serverConfig->port);
                    int len = sizeof(int)+sizeof(long)+item->shmFilename.size()+1;
                    char msg[len];
                    int* portpt = (int*)msg;
                    long* filesizept = (long*)(msg+sizeof(int));
                    char* shmfilename = msg+sizeof(int)+sizeof(long);
                    
                    *portpt = serverConfig->port;
                    *filesizept = item->pfilestat.st_size; //文件大小
                    memcpy(shmfilename, item->shmFilename.c_str(), item->shmFilename.size());
                    msg[len-1] = 0;
                    
                    LOG_DEBUG("msg.port:%d", *portpt);
                    LOG_DEBUG("msg.shmFilenamesize:%d", *filesizept);
                    LOG_DEBUG("msg.shmFilename:%s", shmfilename);                    
                    processMessage.send(msg, len);
                }
                
                delete item;
            }else{
                LOG_WARN("iopump item not find");   
            }
            delete sendfile;
        }
        doneRWIOPumps.clear();
        LOG_DEBUG("sendfileItems.size:%d", sendfileItems.size());
    }
    
    void FileCached::initAsyncClean(){
        asyncClean.set<FileCached, &FileCached::asyncCallback>(this);
        asyncClean.start();
    }
    
    void FileCached::startAsyncClean(RWIOPump* iopump){
        doneRWIOPumps.push_back(iopump);
        asyncClean.send();
    }
    
    
    void FileCached::stopAsyncClean(){
        if(asyncClean.is_active()){
            asyncClean.stop();
        }
    }    
    
    
    bool FileCached::handle(const Event* e){
        LOG_DEBUG("e->getType():%d", e->getType());
        switch(e->getType()){
            case EVENT_TYPE_IO_TRANSPORT:
                return onIOTransportEvent((IOTransportEvent*)e);
        }
        return false;
    }
    
    bool FileCached::onIOTransportEvent(IOTransportEvent* e){
        RWIOPump* ioPumper = (RWIOPump*)e->getSource();
        if(!ioPumper){
            LOG_ERROR("ioPumper is NULL");
        }
        if(e->readEvent->length < 0 || (e->writeEvent && e->writeEvent->length < 0)){ //读出错或写出错(写事件有可能为NULL)
            LOG_ERROR("error, errno:%d", errno);
            ioPumper->close();
            startAsyncClean(ioPumper);
        }else if(e->readEvent->length == 0){ //读结束
            LOG_DEBUG("done.");
            ioPumper->close();
            startAsyncClean(ioPumper);
        }
        return true;
    }
    
    
    bool FileCached::enableCache(size_t cachedfilesize){
        if(cacheConfig->existCacheConfig){
            LOG_DEBUG("cachedfilesize:%d, limit:%d", cachedfilesize, cacheConfig->limit);
            return cachedfilesize < cacheConfig->limit;
        }
        return false;
    }
    
    size_t FileCached::handleCachedFile(CacheConfig* cacheConfig, struct stat& stat, string& filename, std::vector<FileItem*>& files){
        if(handleCachedFile(stat, cacheConfig->base.size(), filename.c_str())){
            //如果存在
            FileItem* fitem = new FileItem();
            fitem->filename.assign(filename);
            fitem->atime = stat.st_atim;
            fitem->filesize = stat.st_size;
            files.push_back(fitem);
            
            return stat.st_size;
        }else{
            delSourceFileEvstat(filename.c_str()+cacheConfig->base.size());
        }
        return 0;
    }
    
    bool FileCached::handleCachedFile(struct stat& stat, int basepathsize, const char* filename){
        const char* srcfilename = filename+basepathsize; //原始文件
        LOG_DEBUG("filename:%s, srcfilename:%s", filename, srcfilename);      
        if(::access(srcfilename, F_OK) != 0){
            //如果文件不存在，就删除cached中的文件
            ::remove(filename);
            return false;
        }else{
            //检查源来文件修改时间，是否与缓存时间相同，如果不同了，就清除缓存文件
            struct stat srcstat;
            if ((lstat(srcfilename, &srcstat)) != 0) {
                ::remove(filename);
                return false;
            }
            
            LOG_DEBUG("srcstat.sec:%d, srcstat.nsec:%ld, stat.sec:%d, stat.nsec:%ld", srcstat.st_ctim.tv_sec, srcstat.st_ctim.tv_nsec, stat.st_ctim.tv_sec, stat.st_ctim.tv_nsec)
            LOG_DEBUG("srcstat.sec:%d, srcstat.nsec:%ld, stat.sec:%d, stat.nsec:%ld", srcstat.st_mtim.tv_sec, srcstat.st_mtim.tv_nsec, stat.st_mtim.tv_sec, stat.st_mtim.tv_nsec)
            if(srcstat.st_mtim.tv_sec != stat.st_mtim.tv_sec ){
                LOG_DEBUG("time not same");
                ::remove(filename);
                return false;
            }
                    
                    
            return true;
        }
    }    
    
    void FileCached::syncCachedFiles(CacheConfig* cacheConfig){
        LOG_DEBUG("cache base : %s", cacheConfig->base.c_str());
        //检查此目录下所有子目录中的文件大小
        vector<FileItem*> files;
        long size = syncCachedFiles(cacheConfig, cacheConfig->base, files);
        LOG_DEBUG("size : %f, capacity: %f, files.size():%d", (double)size/1048576, cacheConfig->capacity, files.size());
        
        if(files.size() == 0){
            return;
        }

        //检查总容量，如果总文件大小大于配置的总容量，就移除长期不访问文件
        if((double)size/1048576 > cacheConfig->capacity){
            //文件太大了点，缓存的文件存多了，需要移除一些长期不访问的文件
            std::sort(files.begin(), files.end(), FileCached::fileItemCompare);
            vector<FileItem*>::iterator it = files.begin();
            for(; it != files.end(); it++){
                FileItem* fitem = (*it);
                LOG_DEBUG("deleted filename:%s", fitem->filename.c_str());
                ::remove(fitem->filename.c_str());
                size -= fitem->filesize;
                if((double)size/1048576 < cacheConfig->capacity){
                    break;
                }
            }
        }


        //移除
        vector<FileItem*>::iterator it = files.begin();
        for(; it != files.end(); it++){
            FileItem* fitem = (*it);
            LOG_DEBUG("fitem->filename:%s", fitem->filename.c_str());
            delete fitem;
        }
        files.clear();
        
        LOG_INFO("size:%d", size);
        cahcedCapacity = (double)size/1048576;
        LOG_INFO("cahcedCapacity:%f", cahcedCapacity);
    }
    
    long FileCached::syncCachedFiles(CacheConfig* cacheConfig, string& filename, std::vector<FileItem*>& files){
        //string& filename = serverConfig->cacheConfig->base;
        struct stat statbuf;
        if ((lstat(filename.c_str(), &statbuf)) != 0) {
            return 0;
        }
        long sum = 0;
        if (S_ISDIR(statbuf.st_mode)) {
                DIR *dir;
                struct dirent *entry;
                dir = opendir(filename.c_str());
                if (!dir) {
                    return sum;
                }
                while ((entry = readdir(dir))) {
                        char *name = entry->d_name;
                        if(!name || name[0] == '.'){
                            continue;
                        }
                        string subfilename = filename;
                        subfilename.append("/").append(entry->d_name);
                        //LOG_DEBUG("subfilename:%s", subfilename.c_str());
                        sum += syncCachedFiles(cacheConfig, subfilename, files);
                }
                closedir(dir);
        }else{
            //sum = statbuf.st_size; //statbuf.st_blocks;
            sum = handleCachedFile(cacheConfig, statbuf, filename, files);

        }
        return sum;
    }    
    
    void FileCached::newSourceFileEvstat(const char* shmfilename, long filesize){
        newSourceFileEvstat(cacheConfig->base.c_str(), shmfilename);
        //检查容量
        cahcedCapacity += ((double)filesize)/1048576;
        
        LOG_INFO("cahcedCapacity:%f", cahcedCapacity);
        
        if(cahcedCapacity > cacheConfig->capacity*1.3){
            syncCachedFiles(cacheConfig);
        }
        
    }
    
    void FileCached::newSourceFileEvstat(const char* basepath, const char* shmfilename){
        const char* srcfilename = shmfilename+strlen(basepath); //原始文件
        LOG_DEBUG("filename:%s, base:%s, srcfilename:%s", shmfilename, basepath, srcfilename);
        LOG_DEBUG("1, sourceFileEvstats.size():%d", sourceFileEvstats.size());
        SourceFileEvstats::iterator it = sourceFileEvstats.find(srcfilename);
        if(it == sourceFileEvstats.end()){
            ev::stat* stat = new ev::stat();
            stat->set<FileCached, &FileCached::statCallback>(this);
            const char* path = strdup(srcfilename);
            LOG_DEBUG("path:%s", path);
            stat->set(path, 10.0);//间隔时间可通过配置
            stat->start();
            filenames.insert(Filenames::value_type(path, strdup(shmfilename)));
            sourceFileEvstats.insert(SourceFileEvstats::value_type(path, stat));
        }
        LOG_DEBUG("2, sourceFileEvstats.size():%d", sourceFileEvstats.size());
    }
    
    
    void FileCached::delSourceFileEvstat(const char* filename){
        LOG_DEBUG("0, filename:%s", filename);
        LOG_DEBUG("0.1, sourceFileEvstats.size():%d", sourceFileEvstats.size());
        SourceFileEvstats::iterator it = sourceFileEvstats.find(filename);
        LOG_DEBUG("0.2");
        if(it != sourceFileEvstats.end()){
            LOG_DEBUG("1");
            //char* key = (char*)it->first;
            ev::stat* stat = it->second;
            sourceFileEvstats.erase(it);
            freeSourceFileEvstat(stat);
        }
    }
    
    void FileCached::freeSourceFileEvstat(ev::stat* stat){
        LOG_DEBUG("2");
        //free(key);
        LOG_DEBUG("3");
        stat->stop();
        LOG_DEBUG("4");
        LOG_DEBUG(stat->path);
        Filenames::iterator it = filenames.find(stat->path);
        if(it != filenames.end()){
            char* shmfilename = it->second;
            filenames.erase(it);
            free(shmfilename);
        }
        
        free((char*)stat->path);
        //free(stat->data);
        
        delete stat;
    }
    
    void FileCached::delAllSourceFileEvstats(){
        SourceFileEvstats::iterator it = sourceFileEvstats.begin();
        for(; it != sourceFileEvstats.end(); it++){
            //delSourceFileEvstat(it->first); //有点费
            freeSourceFileEvstat(it->second);
        }
        sourceFileEvstats.clear();
    }
    
    void FileCached::makeSourceFileEvstats(const char* basepath){
        LOG_DEBUG("cache base : %s", basepath);
        long size = makeSourceFileEvstats(basepath, basepath);
        LOG_DEBUG("size : %f, capacity: %f, sourceFileEvstats.size():%d", (double)size/1048576, cacheConfig->capacity, sourceFileEvstats.size());
        
    }    
    
    long FileCached::makeSourceFileEvstats(const char* basepath, const char* filename){
        //string& filename = serverConfig->cacheConfig->base;
        LOG_DEBUG("filename : %s", filename);
        struct stat statbuf;
        if ((lstat(filename, &statbuf)) != 0) {
            return 0;
        }
        long sum = 0;
        if (S_ISDIR(statbuf.st_mode)) {
                DIR *dir;
                struct dirent *entry;
                dir = opendir(filename);
                if (!dir) {
                    return sum;
                }
                while ((entry = readdir(dir))) {
                        char *name = entry->d_name;
                        if(!name || name[0] == '.'){
                            continue;
                        }
                        string subfilename = filename;
                        subfilename.append("/").append(entry->d_name);
                        //LOG_DEBUG("subfilename:%s", subfilename.c_str());
                        sum += makeSourceFileEvstats(basepath, subfilename.c_str());
                }
                closedir(dir);
        }else{
            if(handleCachedFile(statbuf, strlen(basepath), filename)){
                newSourceFileEvstat(basepath, filename);
                sum = statbuf.st_size;
            }
            
        }
        
        LOG_DEBUG("sum : %d", sum);
        return sum;
    }    
    
    void FileCached::statCallback(ev::stat& stat, int revents){
        const char* srcfilename = stat.path;
        const char* shmfilename = filenames[srcfilename];//(const char*)stat.data; //string(cacheConfig->base.c_str()).append(stat.path);
        LOG_DEBUG("shmfilename: %s, srcfilename:%s", shmfilename, srcfilename);
        if(::access(srcfilename, F_OK) != 0){
            LOG_DEBUG("file not exist, %s", shmfilename);
            LOG_INFO("1 stat.prev.st_size:%d", stat.prev.st_size);
            LOG_INFO("1 stat.attr.st_size:%d", stat.attr.st_size);
            cahcedCapacity -= ((double)stat.prev.st_size)/1048576;
            
            //如果文件不存在，就删除cached中的文件
            if(shmfilename){
                ::remove(shmfilename);
            }else{
                LOG_ERROR("shmfilename not exist!");
            }            
            delSourceFileEvstat(srcfilename);

        }else{
            if(stat.prev.st_mtim.tv_sec != stat.attr.st_mtim.tv_sec){
                LOG_INFO("2 stat.prev.st_size:%d", stat.prev.st_size);
                LOG_INFO("2 stat.attr.st_size:%d", stat.attr.st_size);
                cahcedCapacity -= ((double)stat.prev.st_size)/1048576;
                
                LOG_DEBUG("1 shmfilename:%s, srcstat.sec:%d, srcstat.nsec:%ld, stat.sec:%d, stat.nsec:%ld", shmfilename, stat.prev.st_mtim.tv_sec, stat.prev.st_mtim.tv_nsec, stat.attr.st_mtim.tv_sec, stat.attr.st_mtim.tv_nsec)
                if(shmfilename){
                    ::remove(shmfilename);
                }else{
                    LOG_ERROR("shmfilename not exist!");
                }                        
                delSourceFileEvstat(srcfilename);
            }
        }
    }
    
    
    void FileCached::addCahcedCapacity(long filesize){
        cahcedCapacity += ((double)filesize)/1048576;
    }
    
}






